﻿using System;
using System.Collections;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using WebSpiderLib;

namespace Dictionary.Core
{
    public class Crawler
    {
        public String strUrl { get; set; }
        public String strContent { get; set; }
        /// <summary>
        /// Default constructor
        /// </summary>
        public Crawler()
        {
            this.strUrl = "";
        }
        /// <summary>
        /// Constructor with URL parameter
        /// </summary>
        /// <param name="strUrl"></param>
        public Crawler(String strUrl)
        {
            this.strUrl = strUrl;
        }
        /// <summary>
        /// Method to get Content of WebSite
        /// </summary>
        /// <returns>String</returns>
        private String getContent()
        {
            try
            {
                WebClient client = new WebClient();
                client.Headers.Add("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.2; .NET CLR 1.0.3705;)");
                Stream data = client.OpenRead(this.strUrl);
                StreamReader reader = new StreamReader(data);
                this.strContent = reader.ReadToEnd();
                data.Close();
                reader.Close();
            }
            catch (Exception ex)
            {
                this.strContent = "";
            }
            return this.strContent;
        }
        /// <summary>
        /// Method to get all Source form URL
        /// </summary>
        /// <returns>List<Category></returns>
        public List<Category> getSource()
        {
            List<Category> listResult = new List<Category>();
            try
            {
                if (this.getContent().Length > 0)
                {
                    string strTableContentParttern = "<table.*?border=0.*?width=\"100%\".*?cellpadding=2.*?cellspacing=4>(?<CONTENT>[\\w\\W]*?)</table>";
                    string strLiParttent1 = "/\">(?<LI>[\\w\\W]*?)</a>";
                    string strLiParttent2 = "<a.*?href=\"(?<LI>[\\w\\W]*?)\">";
                    string strDivContent = WebRegEx.GetValueByPattern(strContent, strTableContentParttern, "CONTENT");
                    ArrayList listTitle = WebRegEx.GetRepeatingValuesByPattern(strDivContent, strLiParttent1, "LI");
                    ArrayList listLink = WebRegEx.GetRepeatingValuesByPattern(strDivContent, strLiParttent2, "LI");
                    for (int i = 0; i < listTitle.Count; i++)
                    {
                        Category temp = new Category();
                        temp.title = HtmlRemoval.StripTagsCharArray(listTitle[i].ToString());
                        temp.link = HtmlRemoval.StripTagsCharArray(this.strUrl + listLink[i].ToString());
                        listResult.Add(temp);
                    }

                }
            }
            catch (Exception ex)
            { }
            return listResult;
        }
        /// <summary>
        /// Method to get all category from Source URL
        /// </summary>
        /// <returns>List<Category></returns>
        public List<Category> getCategory()
        {
            List<Category> listResult = new List<Category>();
            try
            {
                if (this.getContent().Length > 0)
                {
                    // get container content
                    string strTableContentParttern = "<div.*?class=\"main vocab\">(?<CONTENT>[\\w\\W]*?)</div></body>";
                    string strDivContent = WebRegEx.GetValueByPattern(strContent, strTableContentParttern, "CONTENT");

                    // get all element <li></li>
                    string strLiContentParttern = "<li>(?<LI>[\\w\\W]*?)(</li>|</lu>)";
                    ArrayList listLiContent = WebRegEx.GetRepeatingValuesByPattern(strDivContent, strLiContentParttern, "LI");
                    
                    for (int i = 0; i < listLiContent.Count; i++)
                    {
                        // get Name and Link
                        string strLiParttent1 = "[\",']>(?<TITLE>[\\w\\W]*?)</a>";
                        string strLiParttent2 = "<a.*?href=[\",'](?<ADDRESS>[\\w\\W]*?)[\",']>";

                        String strTitle = WebRegEx.GetValueByPattern(listLiContent[i].ToString(), strLiParttent1, "TITLE");
                        String strLink = WebRegEx.GetValueByPattern(listLiContent[i].ToString(), strLiParttent2, "ADDRESS");

                        Category temp = new Category();
                        temp.title = HtmlRemoval.StripTagsCharArray(strTitle);
                        temp.link = HtmlRemoval.StripTagsCharArray(this.strUrl + strLink);
                        listResult.Add(temp);
                    }
                }
            }
            catch (Exception ex)
            { }
            return listResult;
        }
        /// <summary>
        /// Method to get all Vocabulary in Category URL
        /// </summary>
        /// <returns></returns>
        public List<Vocabulary> getVocabulary()
        {
            List<Vocabulary> listResult = new List<Vocabulary>();
            try
            {
                if (this.getContent().Length > 0)
                {
                    string strDivContentParttern = "<div.*?class=\"wrapco\">(?<CONTENT>[\\w\\W]*?)<div.*?class=\"foot\"";
                    string strDivContent = WebRegEx.GetValueByPattern(this.strContent, strDivContentParttern, "CONTENT");
                    if (strDivContent == null)
                    {
                        strDivContentParttern = "<div.*?class=\"outerwrap\">(?<CONTENT>[\\w\\W]*?)<div.*?class=\"foot\"";
                        strDivContent = WebRegEx.GetValueByPattern(this.strContent, strDivContentParttern, "CONTENT");
                    }
                    
                    string strLiParttent = "<li>(?<LI>[\\w\\W]*?)</li>";
                    ArrayList listVocab = WebRegEx.GetRepeatingValuesByPattern(strDivContent, strLiParttent, "LI");
                    for (int i = 0; i < listVocab.Count; i++)
                    {
                        Vocabulary temp = new Vocabulary();
                        temp.value = HtmlRemoval.StripTagsCharArray(listVocab[i].ToString());
                        listResult.Add(temp);
                    }
                }
            }
            catch (Exception ex)
            { }
            return listResult;
        }
    }

}
